# coding=utf-8
"""
这是用来调度爬虫的文件
"""
import json

import requests
import time


def get_scrapyd_status():
    url = "http://219.224.166.3:6800/listjobs.json?project=myTest"
    responseJson = json.loads(requests.get(url).content)
    return responseJson


def start_crawl_by_name(spider_name, fm):
    if fm > 62:
        return
    url = "http://219.224.166.3:6800/schedule.json"
    urlParams = {
        "project": "myTest",
        "spider": spider_name,
        "data_block": None
    }
    for i in range(fm, 63):
        list1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        list2 = []
        while len(list1) + len(list2) >= 3:
            time.sleep(10)
            responseJson = get_scrapyd_status()
            # print get_scrapyd_status()
            list1 = responseJson['running']
            print "try to run job ",i
            print "now running "+str(len(list1))+" jobs"
            list2 = responseJson['pending']
        print "push job "+str(i)+" success!"
        time.sleep(10)
        urlParams['data_block'] = i
        print str(i) + ":\n" + requests.post(url, data=urlParams).content


# 以下三行代码分别让服务器爬取学者，文章或者专利信息


# start_crawl_by_name('scholarInfo',13)
# start_crawl_by_name("patentInfo",54)
# start_crawl_by_name("paperInfo", 59)
start_crawl_by_name("morepatentInfo",1)
# start_crawl_by_name("referenceInfo",1)